package com.cw.spider.parse.jxtv;

import cn.wanghaomiao.seimi.annotation.Crawler;
import cn.wanghaomiao.seimi.http.SeimiHttpType;
import cn.wanghaomiao.seimi.spring.common.CrawlerCache;
import cn.wanghaomiao.seimi.struct.Response;
import com.cw.spider.parse.AbstractCrawler;
import com.cw.spider.queue.Myqueue;
import org.jsoup.nodes.Element;
import org.seimicrawler.xpath.JXDocument;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.List;
import java.util.stream.Collectors;

/**
 * 描述: 爬虫 work
 *
 * @author withqianqian@163.com
 * @create 2020-08-27 9:08
 */
@Crawler(name = "jx_basic", httpType = SeimiHttpType.OK_HTTP3, delay = 1,
        httpTimeOut = 3000, queue = Myqueue.class)
public class JxCrawler extends AbstractCrawler {

    @Autowired
    private JxRegionCrawler jxRegionCrawler;
    @Autowired
    private JxBdCrawler jxBdCrawler;

    @Override
    public String[] startUrls() {
        return new String[]{"http://www.jxntv.cn/"};
    }

    @Override
    public void start(Response response) {
        if (response != null) {
            JXDocument doc = response.document();
            if (doc != null) {
                //导航
                List<Object> divs = doc.sel("//div[@class='navbar']/div/div");
                //地区新闻
                Element o = (Element) divs.get(3);
                List<Object> urls = JXDocument.create(o.html()).sel("//div/a/@href");
                List<String> purposeUrls = urls.stream()
                        .map(u -> "http:" + u.toString())
                        .collect(Collectors.toList());
                //底部更多
                List<Object> midNewsUrls = doc.sel("//div[@class='span4']//div[@class='modal-hd']/div[@class='more']/a/@href");
                List<String> strMidNewsUrls = midNewsUrls.stream().map(u -> {
                    if (u.toString().indexOf("http") == -1) {
                        return "http:" + u.toString();
                    } else {
                        return u.toString();
                    }

                }).collect(Collectors.toList());
                purposeUrls.addAll(strMidNewsUrls);

                String[] purposeUrlsArr = purposeUrls.toArray(new String[purposeUrls.size()]);
                jxRegionCrawler.setUrl(purposeUrlsArr);
                CrawlerCache.getCrawlerModel("jx_region_basic").startRequest();


//                //中部新闻+底部新闻
                List<Object> modalBdUrls = doc.sel("//div[@class='modal-bd']//ul[@class='newslist']/li/a/@href");
                List<String> strModalBdUrls = modalBdUrls.stream().map(u -> u.toString()).collect(Collectors.toList());
                String[] strModalBdUrlsArr = strModalBdUrls.toArray(new String[purposeUrls.size()]);
                jxBdCrawler.setUrl(strModalBdUrlsArr);
                CrawlerCache.getCrawlerModel("jx_bd_basic").startRequest();


            }
        }
    }


}
